{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 准备工作"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from lxml.html import fromstring\n",
    "import time,requests,hashlib,base64,json,math,requests_html,os\n",
    "from random import random\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.common.desired_capabilities import DesiredCapabilities\n",
    "from PIL import Image\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 进入知网与文章搜索页"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "opts = webdriver.ChromeOptions()\n",
    "# opts.add_argument('--incognito') # 无痕模式\n",
    "prefs = {'profile.default_content_settings.popups': 0, 'download.default_directory': 'D:\\\\爬虫\\\\知网文章-PDF'}\n",
    "opts.add_experimental_option('prefs', prefs)\n",
    "opts.add_argument('--no-sandbox')#解决DevToolsActivePort文件不存在的报错\n",
    "opts.add_argument('window-size=1920x3000') #指定浏览器分辨率\n",
    "opts.add_argument('--disable-gpu') #谷歌文档提到需要加上一这个属性来规避bug\n",
    "opts.add_argument('--hide-scrollbars') #隐藏滚动条, 应对些特殊页面\n",
    "driver = webdriver.Chrome(options=opts) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get(\"https://kns.cnki.net/kns8/defaultresult/index\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 检查登录状态\n",
    "# element = driver.find_element_by_id('Ecp_loginShowName1') 旧版本写法\n",
    "element = driver.find_element(By.ID, 'Ecp_loginShowName1')\n",
    "login = element.get_attribute('innerHTML')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "高级检索\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# 判断并IP登陆\n",
    "if login != '北京师范大...':\n",
    "    driver.find_element(By.XPATH, '//*[@id=\"Ecp_top_login\"]/a').click()\n",
    "    driver.find_element(By.XPATH, '//*[@id=\"Ecp_Button2\"]/a').click()\n",
    "else:\n",
    "    # 进入高级检索\n",
    "    #element = driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/a[2]')\n",
    "    element = driver.find_element(By.XPATH, '//*[@id=\"ModuleSearch\"]/div[1]/div/div/div[2]/a[2]') \n",
    "    print(element.get_attribute('innerHTML'))\n",
    "    element.click()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 切换窗口\n",
    "driver.switch_to.window(driver.window_handles[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "                        <span>学术期刊</span><em></em>\n"
     ]
    }
   ],
   "source": [
    "# 点击学术期刊\n",
    "element = driver.find_element(By.XPATH, '//*[@id=\"ModuleSearch\"]/div[2]/div/div/ul/li[1]/a') \n",
    "print(element.get_attribute('innerHTML'))\n",
    "\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "专业检索\n"
     ]
    }
   ],
   "source": [
    "# 选择专业检索\n",
    "element = driver.find_element(By.NAME,'majorSearch')\n",
    "print(element.get_attribute('innerHTML'))\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 选择期刊来源\n",
    "\n",
    "## 北大核心来源期刊\n",
    "element = driver.find_element(By.XPATH, '//div[@class=\"extend-tit-labels\"]//input[@key = \"HX\"]')\n",
    "print(element.get_attribute('innerHTML'))\n",
    "element.click()\n",
    "## EI来源期刊\n",
    "element = driver.find_element(By.XPATH, '//div[@class=\"extend-tit-labels\"]//input[@key = \"EI\"]')\n",
    "print(element.get_attribute('innerHTML'))\n",
    "element.click()\n",
    "## CSD来源期刊\n",
    "element = driver.find_element(By.XPATH, '//div[@class=\"extend-tit-labels\"]//input[@key = \"CSD\"]')\n",
    "print(element.get_attribute('innerHTML'))\n",
    "element.click()\n",
    "## SCI来源期刊\n",
    "element = driver.find_element(By.XPATH, '//div[@class=\"extend-tit-labels\"]//input[@key = \"SI\"]')\n",
    "print(element.get_attribute('innerHTML'))\n",
    "element.click()\n",
    "## cssci来源期刊\n",
    "element = driver.find_element(By.XPATH, '//div[@class=\"extend-tit-labels\"]//input[@key = \"CSI\"]')\n",
    "print(element.get_attribute('innerHTML'))\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 设置搜索query\n",
    "# query = 'SU = \"新媒体\" AND (TI = \"人工智能\" OR TI = \"大数据\" OR TI = \"AI\" OR TI = \"big data\" )'\n",
    "query = \"(SU='媒体融合' AND SU = '大数据') OR  (SU='媒体融合' AND SU = '人工智能')\"\n",
    "element = driver.find_element(By.XPATH, '//textarea')\n",
    "element.clear()\n",
    "element.send_keys(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "element = driver.find_element(By.XPATH, '//input[@value =\"检索\"]')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 导出refworks文件"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 自动爬取"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 先确定总页数，以便于规定循环翻页的次数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "文章数量总计: 459\n",
      "结果页数总计: 9.18\n",
      "需要翻10页\n",
      "0\n",
      "当前页数 1\n",
      "50\n",
      "当前页数 2\n",
      "100\n",
      "当前页数 3\n",
      "150\n",
      "当前页数 4\n",
      "200\n",
      "当前页数 5\n",
      "250\n",
      "当前页数 6\n",
      "300\n",
      "当前页数 7\n",
      "350\n",
      "当前页数 8\n",
      "400\n",
      "当前页数 9\n",
      "450\n",
      "当前页数 10\n",
      "459\n",
      "已经全选所有页面文献\n"
     ]
    }
   ],
   "source": [
    "import math\n",
    "import time\n",
    "from random import random\n",
    "import requests_html\n",
    "element = driver.find_element(By.XPATH,'//html')\n",
    "main_content =element.get_attribute('outerHTML')  \n",
    "html = requests_html.HTML(html= main_content, url='https://localhost/')\n",
    "文章数量 = html.xpath('//*[@id=\"countPageDiv\"]/span[1]/em')[0].lxml.text_content()\n",
    "print(\"文章数量总计:\",文章数量)\n",
    "\n",
    "# 选择为50篇\n",
    "driver.find_element(By.XPATH, '//*[@id=\"perPageDiv\"]/div/i').click()\n",
    "time.sleep(3) \n",
    "driver.find_element(By.XPATH, '//*[@id=\"perPageDiv\"]/ul/li[3]').click()\n",
    "time.sleep(3+random()*8)\n",
    "\n",
    "# 需要翻几页\n",
    "结果页数 = int(文章数量)/50\n",
    "print(\"结果页数总计:\",结果页数)\n",
    "pagenum = math.ceil(结果页数)\n",
    "print(\"需要翻\"+str(pagenum)+\"页\")\n",
    "\n",
    "for page in range(1,pagenum+1):\n",
    "#     element = driver.find_element_by_xpath('//html')\n",
    "#     main_content =element.get_attribute('outerHTML')  \n",
    "#     html = requests_html.HTML(html= main_content, url='https://localhost/')\n",
    "    已选文献 = driver.find_element(By.XPATH,'//*[@id=\"selectCount\"]').get_attribute('innerHTML')\n",
    "    print(int(已选文献))\n",
    "    当前页数_element = driver.find_element(By.XPATH,'//*[@id=\"countPageDiv\"]/span[2]').get_attribute('innerHTML')\n",
    "    当前页数 = int(当前页数_element.split('/')[0])\n",
    "    print(\"当前页数\",当前页数)\n",
    "    # Alert Text: 不能超过500个,若要重新选择,请按清除按钮,再进行选取操作\n",
    "    if int(当前页数) < pagenum:\n",
    "        # 勾选本页文献\n",
    "        time.sleep(3+random()*8)\n",
    "        driver.find_element(By.XPATH,'//*[@id=\"briefBox\"]/div[1]/div/div[2]/div[1]/label').click()\n",
    "        # 翻页\n",
    "#         element = driver.find_element_by_xpath('//html')\n",
    "#         main_content =element.get_attribute('outerHTML')  \n",
    "#         main_content\n",
    "        a = driver.find_element(By.XPATH,'//*[@id=\"PageNext\"]')\n",
    "        a.click()\n",
    "        # 等待时间 平均时间38*103\n",
    "        time.sleep(3+random()*8)\n",
    "    else:\n",
    "        # 勾选本页文献\n",
    "        time.sleep(3+random()*8)\n",
    "        driver.find_element(By.XPATH,'//*[@id=\"briefBox\"]/div[1]/div/div[2]/div[1]/label').click()\n",
    "        已选文献 = driver.find_element(By.XPATH,'//*[@id=\"selectCount\"]').get_attribute('innerHTML')\n",
    "        print(int(已选文献))\n",
    "        print(\"已经全选所有页面文献\")\n",
    "        element = driver.find_element(By.XPATH,'//html')\n",
    "        main_content =element.get_attribute('outerHTML') \n",
    "        main_content\n",
    "        # 选择“导出与分析”\n",
    "        driver.find_element(By.XPATH,'//*[@id=\"batchOpsBox\"]/li[2]').click()\n",
    "        # 选择“导出文件”\n",
    "        time.sleep(3)\n",
    "        driver.find_element(By.XPATH,'//*[@id=\"batchOpsBox\"]/li[2]/ul/li[1]').click()\n",
    "        # 选择Refworks格式\n",
    "        driver.find_element(By.XPATH,'//*[@id=\"batchOpsBox\"]/li[2]/ul/li[1]/ul/li[8]').click()\n",
    "        # ----------------------------进入导出页面------------------------------------\n",
    "        driver.switch_to.window(driver.window_handles[2])\n",
    "        time.sleep(3+random()*8)\n",
    "        driver.find_element(By.XPATH,'/html/body/div[3]/div/div[1]/ul/li[8]').click()\n",
    "        driver.find_element(By.XPATH,'//*[@id=\"litotxt\"]').click()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 文章下载——PDF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 测试（可跳过）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "ename": "NoSuchElementException",
     "evalue": "Message: no such element: Unable to locate element: {\"method\":\"xpath\",\"selector\":\"//*[@id=\"gridTable\"]/table/tbody/tr[8]/td[2]/a\"}\n  (Session info: chrome=118.0.5993.71); For documentation on this error, please visit: https://www.selenium.dev/documentation/webdriver/troubleshooting/errors#no-such-element-exception\nStacktrace:\n\tGetHandleVerifier [0x00007FF733228EF2+54786]\n\t(No symbol) [0x00007FF733195612]\n\t(No symbol) [0x00007FF73304A64B]\n\t(No symbol) [0x00007FF73308B79C]\n\t(No symbol) [0x00007FF73308B91C]\n\t(No symbol) [0x00007FF7330C6D87]\n\t(No symbol) [0x00007FF7330ABEAF]\n\t(No symbol) [0x00007FF7330C4D02]\n\t(No symbol) [0x00007FF7330ABC43]\n\t(No symbol) [0x00007FF733080941]\n\t(No symbol) [0x00007FF733081B84]\n\tGetHandleVerifier [0x00007FF733577F52+3524194]\n\tGetHandleVerifier [0x00007FF7335CD800+3874576]\n\tGetHandleVerifier [0x00007FF7335C5D7F+3843215]\n\tGetHandleVerifier [0x00007FF7332C5086+694166]\n\t(No symbol) [0x00007FF7331A0A88]\n\t(No symbol) [0x00007FF73319CA94]\n\t(No symbol) [0x00007FF73319CBC2]\n\t(No symbol) [0x00007FF73318CC83]\n\tBaseThreadInitThunk [0x00007FFC71C6257D+29]\n\tRtlUserThreadStart [0x00007FFC73EAAA58+40]\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNoSuchElementException\u001b[0m                    Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[19], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;66;03m## 点击谋篇文章标题\u001b[39;00m\n\u001b[1;32m----> 2\u001b[0m \u001b[43mdriver\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfind_element\u001b[49m\u001b[43m(\u001b[49m\u001b[43mBy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mXPATH\u001b[49m\u001b[43m,\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m//*[@id=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mgridTable\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m]/table/tbody/tr[8]/td[2]/a\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mclick()\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py:741\u001b[0m, in \u001b[0;36mWebDriver.find_element\u001b[1;34m(self, by, value)\u001b[0m\n\u001b[0;32m    738\u001b[0m     by \u001b[38;5;241m=\u001b[39m By\u001b[38;5;241m.\u001b[39mCSS_SELECTOR\n\u001b[0;32m    739\u001b[0m     value \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m[name=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mvalue\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m]\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m--> 741\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[43mCommand\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mFIND_ELEMENT\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43musing\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mby\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mvalue\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m}\u001b[49m\u001b[43m)\u001b[49m[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mvalue\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py:347\u001b[0m, in \u001b[0;36mWebDriver.execute\u001b[1;34m(self, driver_command, params)\u001b[0m\n\u001b[0;32m    345\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcommand_executor\u001b[38;5;241m.\u001b[39mexecute(driver_command, params)\n\u001b[0;32m    346\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m response:\n\u001b[1;32m--> 347\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43merror_handler\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcheck_response\u001b[49m\u001b[43m(\u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    348\u001b[0m     response[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mvalue\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_unwrap_value(response\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mvalue\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m))\n\u001b[0;32m    349\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m response\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\selenium\\webdriver\\remote\\errorhandler.py:229\u001b[0m, in \u001b[0;36mErrorHandler.check_response\u001b[1;34m(self, response)\u001b[0m\n\u001b[0;32m    227\u001b[0m         alert_text \u001b[38;5;241m=\u001b[39m value[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124malert\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m    228\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m exception_class(message, screen, stacktrace, alert_text)  \u001b[38;5;66;03m# type: ignore[call-arg]  # mypy is not smart enough here\u001b[39;00m\n\u001b[1;32m--> 229\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m exception_class(message, screen, stacktrace)\n",
      "\u001b[1;31mNoSuchElementException\u001b[0m: Message: no such element: Unable to locate element: {\"method\":\"xpath\",\"selector\":\"//*[@id=\"gridTable\"]/table/tbody/tr[8]/td[2]/a\"}\n  (Session info: chrome=118.0.5993.71); For documentation on this error, please visit: https://www.selenium.dev/documentation/webdriver/troubleshooting/errors#no-such-element-exception\nStacktrace:\n\tGetHandleVerifier [0x00007FF733228EF2+54786]\n\t(No symbol) [0x00007FF733195612]\n\t(No symbol) [0x00007FF73304A64B]\n\t(No symbol) [0x00007FF73308B79C]\n\t(No symbol) [0x00007FF73308B91C]\n\t(No symbol) [0x00007FF7330C6D87]\n\t(No symbol) [0x00007FF7330ABEAF]\n\t(No symbol) [0x00007FF7330C4D02]\n\t(No symbol) [0x00007FF7330ABC43]\n\t(No symbol) [0x00007FF733080941]\n\t(No symbol) [0x00007FF733081B84]\n\tGetHandleVerifier [0x00007FF733577F52+3524194]\n\tGetHandleVerifier [0x00007FF7335CD800+3874576]\n\tGetHandleVerifier [0x00007FF7335C5D7F+3843215]\n\tGetHandleVerifier [0x00007FF7332C5086+694166]\n\t(No symbol) [0x00007FF7331A0A88]\n\t(No symbol) [0x00007FF73319CA94]\n\t(No symbol) [0x00007FF73319CBC2]\n\t(No symbol) [0x00007FF73318CC83]\n\tBaseThreadInitThunk [0x00007FFC71C6257D+29]\n\tRtlUserThreadStart [0x00007FFC73EAAA58+40]\n"
     ]
    }
   ],
   "source": [
    "## 点击谋篇文章标题\n",
    "driver.find_element(By.XPATH,'//*[@id=\"gridTable\"]/table/tbody/tr[8]/td[2]/a').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 切换窗口\n",
    "driver.switch_to.window(driver.window_handles[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 点击pdf下载\n",
    "driver.find_element(By.XPATH,'//*[@id=\"pdfDown\"]').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 若有验证码，则弹出新页面--->换窗口\n",
    "driver.switch_to.window(driver.window_handles[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 对验证码进行截图\n",
    "# encoding:utf-8\n",
    "from PIL import Image\n",
    "from selenium import webdriver\n",
    "import requests\n",
    "import time\n",
    "import hashlib\n",
    "import base64\n",
    "import json\n",
    " \n",
    "# url = 'https://weixin.sogou.com/antispider/?from=http%3A%2F%2Fweixin.sogou.com%2Fweixin%3Ftype%3D2%26query%3Dpython'\n",
    "# driver = webdriver.Chrome()\n",
    "# driver.maximize_window()  # 将浏览器最大化\n",
    "# driver.get(url)\n",
    "# # 截取当前网页并放到D盘下命名为printscreen，该网页有我们需要的验证码\n",
    "driver.save_screenshot('C:\\\\Users\\\\zjl\\\\Desktop\\\\web数据挖掘\\\\web_img\\\\web_imgprintscreen.png')\n",
    "imgelement = driver.find_element_by_id('vImg')  # 定位验证码\n",
    "location = imgelement.location  # 获取验证码x,y轴坐标\n",
    "print(location)\n",
    "size = imgelement.size  # 获取验证码的长宽\n",
    "print(size)\n",
    "rangle = (int(location['x']+260), int(location['y']+135), int(location['x'] + size['width']+360),\n",
    "          int(location['y'] + size['height']+170))  # 写成我们需要截取的位置坐标\n",
    "i = Image.open(\"C:\\\\Users\\\\zjl\\\\Desktop\\\\web数据挖掘\\\\web_img\\\\web_imgprintscreen.png\")  # 打开截图\n",
    "frame4 = i.crop(rangle)  # 使用Image的crop函数，从截图中再次截取我们需要的区域\n",
    "frame4=frame4.convert('RGB')\n",
    "frame4.save('C:\\\\Users\\\\zjl\\\\Desktop\\\\web数据挖掘\\\\web_img\\\\save.jpg') # 保存我们接下来的验证码图片 进行打码\n",
    " \n",
    "# driver.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 讯飞OCR识别验证码\n",
    "\"\"\"\n",
    "  印刷文字识别WebAPI接口调用示例接口文档(必看)：https://doc.xfyun.cn/rest_api/%E5%8D%B0%E5%88%B7%E6%96%87%E5%AD%97%E8%AF%86%E5%88%AB.html\n",
    "  上传图片base64编码后进行urlencode要求base64编码和urlencode后大小不超过4M最短边至少15px，最长边最大4096px支持jpg/png/bmp格式\n",
    "  (Very Important)创建完webapi应用添加合成服务之后一定要设置ip白名单，找到控制台--我的应用--设置ip白名单，如何设置参考：http://bbs.xfyun.cn/forum.php?mod=viewthread&tid=41891\n",
    "  错误码链接：https://www.xfyun.cn/document/error-code (code返回错误码时必看)\n",
    "  @author iflytek\n",
    "\"\"\"\n",
    "#-*- coding: utf-8 -*-\n",
    "import requests\n",
    "import time\n",
    "import hashlib\n",
    "import base64\n",
    "import json\n",
    "#from urllib import parse\n",
    "# 印刷文字识别 webapi 接口地址\n",
    "URL = \"http://webapi.xfyun.cn/v1/service/v1/ocr/general\"\n",
    "# 应用ID (必须为webapi类型应用，并印刷文字识别服务，参考帖子如何创建一个webapi应用：http://bbs.xfyun.cn/forum.php?mod=viewthread&tid=36481)\n",
    "APPID = \"600a7fb3\"\n",
    "# 接口密钥(webapi类型应用开通印刷文字识别服务后，控制台--我的应用---印刷文字识别---服务的apikey)\n",
    "API_KEY = \"73071c3d054a8bfc81f2ef13debafb91\"\n",
    "def getHeader():\n",
    "#  当前时间戳\n",
    "    curTime = str(int(time.time()))\n",
    "#  支持语言类型和是否开启位置定位(默认否)\n",
    "    param = {\"language\": \"cn|en\", \"location\": \"false\"}\n",
    "    param = json.dumps(param)\n",
    "    paramBase64 = base64.b64encode(param.encode('utf-8'))\n",
    "\n",
    "    m2 = hashlib.md5()\n",
    "    str1 = API_KEY + curTime + str(paramBase64,'utf-8')\n",
    "    m2.update(str1.encode('utf-8'))\n",
    "    checkSum = m2.hexdigest()\n",
    "# 组装http请求头\n",
    "    header = {\n",
    "        'X-CurTime': curTime,\n",
    "        'X-Param': paramBase64,\n",
    "        'X-Appid': APPID,\n",
    "        'X-CheckSum': checkSum,\n",
    "        'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8',\n",
    "    }\n",
    "    return header\n",
    "#上传文件并进行base64位编码\n",
    "with open(r'C:\\\\Users\\\\zjl\\\\Desktop\\\\web数据挖掘\\\\web_img\\\\save.jpg', 'rb') as f:\n",
    "    f1 = f.read()\n",
    "\n",
    "f1_base64 = str(base64.b64encode(f1), 'utf-8')\n",
    "\n",
    "# ## 网络外链转换base64位编码\n",
    "\n",
    "# f2_base64 = str(base64.b64encode(base64data), 'utf-8')\n",
    "\n",
    "\n",
    "data = {\n",
    "        'image':f1_base64\n",
    "        }\n",
    "\n",
    "\n",
    "\n",
    "r = requests.post(URL, data=data, headers=getHeader())\n",
    "result = str(r.content, 'utf-8')\n",
    "# # 错误码链接：https://www.xfyun.cn/document/error-code (code返回错误码时必看)\n",
    "# print(result)\n",
    "# input(\"Entry the any key to exit\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 观察返回值\n",
    "r.json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 若有OCR返回值，得到验证码\n",
    "yzm = r.json()['data']['block'][0]['line'][0]['word'][0]['content'()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 输入验证码\n",
    "time.sleep(3)\n",
    "driver.find_element_by_xpath('//*[@id=\"vcode\"]').send_keys(yzm)\n",
    "## 提交\n",
    "time.sleep(3)\n",
    "driver.find_element_by_xpath('/html/body/div/form/dl/dd/button').click()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 自动爬取"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 第一步：获取全部URL队列 \n",
    "* 尽量减少循环步骤\n",
    "* 尽量减少打开网页的次数（网络不稳定等不可抗力）\n",
    "* 便于爬取大量文章时出现中断，继续爬取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 切换窗口\n",
    "driver.switch_to.window(driver.window_handles[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 获取链接的函数封装\n",
    "def get_link():\n",
    "    html=driver.page_source\n",
    "    root=fromstring(html)\n",
    "    href = [x for x in root.xpath('.//*[@id=\"gridTable\"]/table/tbody/tr/td[2]/a/@href')]\n",
    "    c_link=['https://kns.cnki.net/kcms/detail/detail.aspx?'+x.split('&')[6]+'&'+x.split('&')[5]+'&'+x.split('&')[4] for x in href]\n",
    "    return c_link\n",
    "    # del c_link[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "文章数量总计: 459\n",
      "结果页数总计: 9.18\n",
      "需要翻10页\n"
     ]
    }
   ],
   "source": [
    "import math\n",
    "import time\n",
    "from random import random\n",
    "import requests_html\n",
    "element = driver.find_element(By.XPATH,'//html')\n",
    "main_content =element.get_attribute('outerHTML')  \n",
    "html = requests_html.HTML(html= main_content, url='https://localhost/')\n",
    "文章数量 = html.xpath('//*[@id=\"countPageDiv\"]/span[1]/em')[0].lxml.text_content()\n",
    "print(\"文章数量总计:\",文章数量)\n",
    "\n",
    "# 选择为50篇\n",
    "driver.find_element(By.XPATH, '//*[@id=\"perPageDiv\"]/div/i').click()\n",
    "time.sleep(3) \n",
    "driver.find_element(By.XPATH, '//*[@id=\"perPageDiv\"]/ul/li[3]').click()\n",
    "time.sleep(3+random()*8)\n",
    "\n",
    "# 需要翻几页\n",
    "结果页数 = int(文章数量)/50\n",
    "print(\"结果页数总计:\",结果页数)\n",
    "pagenum = math.ceil(结果页数)\n",
    "print(\"需要翻\"+str(pagenum)+\"页\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n"
     ]
    }
   ],
   "source": [
    "all_link = []\n",
    "use_index = []\n",
    "\n",
    "\n",
    "for page in range(1,pagenum+1):\n",
    "    print(page)\n",
    "    time.sleep(3)\n",
    "    k = get_link()\n",
    "    lock_index = []\n",
    "    # print(len(k))\n",
    "    for i in range(1,51): ##50篇一页or20篇一页\n",
    "        \n",
    "        driver.switch_to.window(driver.window_handles[1]) #切换窗口\n",
    "        down_xpath = '//*[@id=\"gridTable\"]/div/div/table/tbody/tr['+str(i)+']/td[8]'\n",
    "        # d = driver.find_element(By.XPATH,down_xpath).get_attribute('innerHTML')\n",
    "        # print(\"翻页完成\")\n",
    "        try:\n",
    "            d = driver.find_element(By.XPATH,down_xpath).get_attribute('innerHTML')\n",
    "        except:\n",
    "            print(\"翻页完成\")\n",
    "        if len(d)>700:  ##说明文章未被锁，正常  \n",
    "            a1 = i-1\n",
    "#             use_index.append[a1]\n",
    "        else:\n",
    "            a2 = i-1\n",
    "            lock_index.append(a2)\n",
    "            print(page,lock_index)\n",
    "            for j in lock_index:\n",
    "                del k[j-1]\n",
    "    all_link.extend(k)\n",
    "    page += 1   \n",
    "    try:\n",
    "        driver.find_element(By.XPATH,'//*[@id=\"PageNext\"]').click()\n",
    "    except:\n",
    "        print(\"_________________________________\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = {\"文献详细页链接\":all_link}\n",
    "data = pd.DataFrame(data)\n",
    "data.to_excel(\"链接.xlsx\",sheet_name=\"sheetname\",index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 第二步：下载PDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = pd.read_excel('链接.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "381"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url_list = url[\"文献详细页链接\"].tolist()\n",
    "len(url_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 封装函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "def down_PDF():\n",
    "    driver.find_element(By.XPATH,'//*[@id=\"pdfDown\"]').click()\n",
    "    time.sleep(30)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def switch_win():\n",
    "    time.sleep(5)\n",
    "    now_page = len(driver.window_handles)\n",
    "    print(\"当前页面数\",now_page)\n",
    "    driver.switch_to.window(driver.window_handles[now_page-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "def img_isTrue():\n",
    "    flag=True\n",
    "    try:\n",
    "        driver.find_element(By.XPATH,'//*[@id=\"vImg\"]')\n",
    "        return flag\n",
    "    except:\n",
    "        flag=False\n",
    "        return flag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 尝试用图鉴\n",
    "\n",
    "import base64\n",
    "import json\n",
    "import requests\n",
    "\n",
    "def base64_api(uname, pwd, img, typeid):\n",
    "    with open(img, 'rb') as f:\n",
    "        base64_data = base64.b64encode(f.read())\n",
    "        b64 = base64_data.decode()\n",
    "    data = {\"username\": uname, \"password\": pwd, \"typeid\": typeid, \"image\": b64}\n",
    "    result = json.loads(requests.post(\"http://api.ttshitu.com/predict\", json=data).text)\n",
    "    if result['success']:\n",
    "        return result[\"data\"][\"result\"]\n",
    "    else:\n",
    "        return result[\"message\"]\n",
    "    return \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "def OCR():\n",
    "    time.sleep(5)\n",
    "    # 截图\n",
    "    element=driver.find_element_by_xpath('//*[@id=\"vImg\"]')\n",
    "    # 选择存储路径\n",
    "    element.screenshot('D:\\\\jupyter-python\\\\web-data-mining-final-cnki-master\\\\web_img\\\\save.jpg')\n",
    "    # api\n",
    "    yzm = base64_api(uname='trevor', pwd='18078846977', img=\"D:\\\\jupyter-python\\\\web-data-mining-final-cnki-master\\\\web_img\\\\save.jpg\", typeid=3)\n",
    "    isalnum_yzm = yzm.isalnum()\n",
    "    if len(yzm) == 4 and isalnum_yzm == True:\n",
    "        # 填写/提交验证码\n",
    "        time.sleep(5)\n",
    "        driver.find_element_by_xpath('//*[@id=\"vcode\"]').send_keys(yzm)\n",
    "        print(\"可提交的验证码：\",yzm)\n",
    "        time.sleep(5)\n",
    "        driver.find_element_by_xpath('/html/body/div/form/dl/dd/button').click()\n",
    "#             pop_window_isTrue()\n",
    "        results = True\n",
    "        time.sleep(30)\n",
    "    else:\n",
    "        # 点击验证码图片\n",
    "        time.sleep(5)\n",
    "        print(\"换验证码\")\n",
    "        driver.find_element_by_xpath('//*[@id=\"vImg\"]').click() ##换验证码\n",
    "        isimgTrue = True\n",
    "        results = False\n",
    "\n",
    "    return results     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_file():\n",
    "    E = os.listdir(\"D:\\\\jupyter-python\\\\web-data-mining-final-cnki-master\\\\自动下载PDF\")\n",
    "    file_nmb = len(E)\n",
    "    return file_nmb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ## 加载时间过长\n",
    "def timeout():\n",
    "    print(\"进行超时处理\")\n",
    "    driver.set_page_load_timeout(10)\n",
    "    while True:\n",
    "        try:\n",
    "            new_url = driver.current_url\n",
    "            driver.get(new_url)\n",
    "            break\n",
    "        except:\n",
    "            pass  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 判断是否出现弹窗\n",
    "## 代码优化后 这个函数可以不用到\n",
    "def pop_window_isTrue():\n",
    "    try:\n",
    "        driver.switch_to.alert\n",
    "        print(\"出现弹窗\")\n",
    "        return True\n",
    "        \n",
    "    except:\n",
    "        print(\"未出现弹窗\")\n",
    "        return False\n",
    "        pass\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "未出现弹窗\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pop_window_isTrue()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 自动下载"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "opts = webdriver.ChromeOptions()\n",
    "# opts.add_argument('--incognito') # 无痕模式\n",
    "prefs = {'profile.default_content_settings.popups': 0, 'download.default_directory': 'D:\\\\jupyter-python\\\\web-data-mining-final-cnki-master\\\\自动下载PDF'}\n",
    "opts.add_experimental_option('prefs', prefs)\n",
    "opts.add_argument('--no-sandbox')#解决DevToolsActivePort文件不存在的报错\n",
    "opts.add_argument('window-size=1920x3000') #指定浏览器分辨率\n",
    "opts.add_argument('--disable-gpu') #谷歌文档提到需要加上一这个属性来规避bug\n",
    "opts.add_argument('--hide-scrollbars') #隐藏滚动条, 应对些特殊页面\n",
    "# driver = webdriver.Chrome(options=opts) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "opts.add_argument(\"start-maximized\")\n",
    "opts.add_argument(\"enable-automation\")\n",
    "opts.add_argument(\"--disable-infobars\")\n",
    "opts.add_argument(\"--disable-dev-shm-usage\")\n",
    "opts.add_argument(\"--disable-browser-side-navigation\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 1\n",
      "进行超时处理\n",
      "https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ202110047&dbname=CJFDAUTO\n",
      "进行超时处理\n",
      "hov_file_nmb: 138\n",
      "进行超时处理\n",
      "file_nmb: 139\n",
      "isDownLoadTrue： True\n",
      "下载成功，文章数量增加\n",
      "关闭页面，1.i+1\n",
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 2\n",
      "进行超时处理\n",
      "https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DXCB202103007&dbname=CJFDLAST2021\n",
      "进行超时处理\n",
      "hov_file_nmb: 139\n",
      "进行超时处理\n",
      "file_nmb: 140\n",
      "isDownLoadTrue： True\n",
      "下载成功，文章数量增加\n",
      "关闭页面，1.i+1\n",
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 3\n",
      "进行超时处理\n",
      "https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202110012&dbname=CJFDLAST2021\n",
      "进行超时处理\n",
      "hov_file_nmb: 140\n",
      "进行超时处理\n",
      "file_nmb: 141\n",
      "isDownLoadTrue： True\n",
      "下载成功，文章数量增加\n",
      "关闭页面，1.i+1\n",
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 4\n",
      "进行超时处理\n",
      "https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DXCB202103007&dbname=CJFDAUTO\n",
      "进行超时处理\n",
      "hov_file_nmb: 141\n",
      "进行超时处理\n",
      "file_nmb: 142\n",
      "isDownLoadTrue： True\n",
      "下载成功，文章数量增加\n",
      "关闭页面，1.i+1\n",
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 5\n",
      "进行超时处理\n",
      "https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202110012&dbname=CJFDAUTO\n",
      "进行超时处理\n",
      "hov_file_nmb: 142\n",
      "进行超时处理\n",
      "file_nmb: 143\n",
      "isDownLoadTrue： True\n",
      "下载成功，文章数量增加\n",
      "关闭页面，1.i+1\n",
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 6\n",
      "进行超时处理\n",
      "https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CBFX202105009&dbname=CJFDLAST2021\n",
      "进行超时处理\n",
      "hov_file_nmb: 143\n",
      "进行超时处理\n",
      "file_nmb: 144\n",
      "isDownLoadTrue： True\n",
      "下载成功，文章数量增加\n",
      "关闭页面，1.i+1\n",
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 7\n",
      "进行超时处理\n",
      "https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CBFX202105009&dbname=CJFDAUTO\n",
      "进行超时处理\n",
      "hov_file_nmb: 144\n",
      "进行超时处理\n",
      "file_nmb: 145\n",
      "isDownLoadTrue： True\n",
      "下载成功，文章数量增加\n",
      "关闭页面，1.i+1\n",
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 8\n",
      "进行超时处理\n",
      "https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202109018&dbname=CJFDLAST2021\n",
      "进行超时处理\n",
      "hov_file_nmb: 145\n",
      "进行超时处理\n",
      "file_nmb: 146\n",
      "isDownLoadTrue： True\n",
      "下载成功，文章数量增加\n",
      "关闭页面，1.i+1\n",
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 9\n",
      "进行超时处理\n",
      "https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202109018&dbname=CJFDAUTO\n",
      "进行超时处理\n",
      "hov_file_nmb: 146\n",
      "进行超时处理\n",
      "file_nmb: 147\n",
      "isDownLoadTrue： True\n",
      "下载成功，文章数量增加\n",
      "关闭页面，1.i+1\n",
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 10\n",
      "进行超时处理\n",
      "https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=BJZY202105005&dbname=CJFDLAST2021\n",
      "进行超时处理\n",
      "hov_file_nmb: 147\n",
      "进行超时处理\n",
      "file_nmb: 148\n",
      "isDownLoadTrue： True\n",
      "下载成功，文章数量增加\n",
      "关闭页面，1.i+1\n",
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 11\n",
      "进行超时处理\n",
      "https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CBGJ202108022&dbname=CJFDLAST2021\n",
      "进行超时处理\n",
      "hov_file_nmb: 148\n",
      "进行超时处理\n",
      "file_nmb: 149\n",
      "isDownLoadTrue： True\n",
      "下载成功，文章数量增加\n",
      "关闭页面，1.i+1\n",
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 12\n",
      "进行超时处理\n",
      "https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CBGJ202108022&dbname=CJFDAUTO\n",
      "进行超时处理\n",
      "hov_file_nmb: 149\n",
      "进行超时处理\n",
      "file_nmb: 150\n",
      "isDownLoadTrue： True\n",
      "下载成功，文章数量增加\n",
      "关闭页面，1.i+1\n",
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 13\n",
      "进行超时处理\n",
      "https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ202107034&dbname=CJFDLAST2021\n",
      "进行超时处理\n",
      "hov_file_nmb: 150\n",
      "进行超时处理\n",
      "file_nmb: 151\n",
      "isDownLoadTrue： True\n",
      "下载成功，文章数量增加\n",
      "关闭页面，1.i+1\n",
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 14\n",
      "进行超时处理\n",
      "https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ202107034&dbname=CJFDAUTO\n",
      "进行超时处理\n",
      "hov_file_nmb: 151\n",
      "进行超时处理\n",
      "file_nmb: 152\n",
      "isDownLoadTrue： True\n",
      "下载成功，文章数量增加\n",
      "关闭页面，1.i+1\n",
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 15\n",
      "进行超时处理\n",
      "https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202107013&dbname=CJFDLAST2021\n",
      "进行超时处理\n",
      "hov_file_nmb: 152\n",
      "进行超时处理\n",
      "file_nmb: 153\n",
      "isDownLoadTrue： True\n",
      "下载成功，文章数量增加\n",
      "关闭页面，1.i+1\n",
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 16\n",
      "进行超时处理\n",
      "https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202107002&dbname=CJFDLAST2021\n",
      "进行超时处理\n",
      "hov_file_nmb: 153\n",
      "进行超时处理\n",
      "file_nmb: 154\n",
      "isDownLoadTrue： True\n",
      "下载成功，文章数量增加\n",
      "关闭页面，1.i+1\n",
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 17\n",
      "进行超时处理\n",
      "https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=BJZG202104003&dbname=CJFDLAST2021\n",
      "进行超时处理\n",
      "hov_file_nmb: 154\n",
      "进行超时处理\n",
      "file_nmb: 155\n",
      "isDownLoadTrue： True\n",
      "下载成功，文章数量增加\n",
      "关闭页面，1.i+1\n",
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 18\n",
      "进行超时处理\n",
      "https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=BJZG202104008&dbname=CJFDLAST2021\n",
      "进行超时处理\n",
      "hov_file_nmb: 155\n",
      "进行超时处理\n",
      "file_nmb: 156\n",
      "isDownLoadTrue： True\n",
      "下载成功，文章数量增加\n",
      "关闭页面，1.i+1\n",
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 19\n",
      "进行超时处理\n",
      "https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202106011&dbname=CJFDLAST2021\n",
      "进行超时处理\n",
      "hov_file_nmb: 156\n",
      "进行超时处理\n",
      "file_nmb: 157\n",
      "isDownLoadTrue： True\n",
      "下载成功，文章数量增加\n",
      "关闭页面，1.i+1\n",
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 20\n",
      "进行超时处理\n",
      "https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ202105013&dbname=CJFDLAST2021\n",
      "进行超时处理\n",
      "hov_file_nmb: 157\n",
      "进行超时处理\n",
      "file_nmb: 158\n",
      "isDownLoadTrue： True\n",
      "下载成功，文章数量增加\n",
      "关闭页面，1.i+1\n",
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 21\n",
      "进行超时处理\n",
      "https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ202105031&dbname=CJFDLAST2021\n",
      "进行超时处理\n",
      "hov_file_nmb: 158\n",
      "进行超时处理\n",
      "file_nmb: 159\n",
      "isDownLoadTrue： True\n",
      "下载成功，文章数量增加\n",
      "关闭页面，1.i+1\n",
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 22\n",
      "进行超时处理\n",
      "https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202105014&dbname=CJFDLAST2021\n",
      "进行超时处理\n",
      "hov_file_nmb: 159\n",
      "进行超时处理\n",
      "file_nmb: 160\n",
      "isDownLoadTrue： True\n",
      "下载成功，文章数量增加\n",
      "关闭页面，1.i+1\n",
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 23\n",
      "进行超时处理\n",
      "https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DDDS202103029&dbname=CJFDLAST2021\n",
      "进行超时处理\n",
      "hov_file_nmb: 160\n",
      "进行超时处理\n",
      "file_nmb: 161\n",
      "isDownLoadTrue： True\n",
      "下载成功，文章数量增加\n",
      "关闭页面，1.i+1\n",
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 24\n",
      "进行超时处理\n",
      "https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202104021&dbname=CJFDLAST2021\n",
      "进行超时处理\n",
      "hov_file_nmb: 161\n",
      "进行超时处理\n",
      "file_nmb: 162\n",
      "isDownLoadTrue： True\n",
      "下载成功，文章数量增加\n",
      "关闭页面，1.i+1\n",
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 25\n",
      "进行超时处理\n",
      "https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=ZGCB202104001&dbname=CJFDLAST2021\n",
      "进行超时处理\n",
      "hov_file_nmb: 162\n",
      "进行超时处理\n",
      "file_nmb: 163\n",
      "isDownLoadTrue： True\n",
      "下载成功，文章数量增加\n",
      "关闭页面，1.i+1\n",
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 26\n",
      "进行超时处理\n",
      "https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=ZGCB202104003&dbname=CJFDLAST2021\n",
      "进行超时处理\n",
      "hov_file_nmb: 163\n",
      "进行超时处理\n",
      "file_nmb: 164\n",
      "isDownLoadTrue： True\n",
      "下载成功，文章数量增加\n",
      "关闭页面，1.i+1\n",
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 27\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[60], line 10\u001b[0m\n\u001b[0;32m      8\u001b[0m driver \u001b[38;5;241m=\u001b[39m webdriver\u001b[38;5;241m.\u001b[39mChrome(options\u001b[38;5;241m=\u001b[39mopts) \n\u001b[0;32m      9\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m3\u001b[39m)\n\u001b[1;32m---> 10\u001b[0m \u001b[43mdriver\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl_list\u001b[49m\u001b[43m[\u001b[49m\u001b[43mi\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     11\u001b[0m timeout()\n\u001b[0;32m     12\u001b[0m \u001b[38;5;28mprint\u001b[39m(url_list[i])\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py:356\u001b[0m, in \u001b[0;36mWebDriver.get\u001b[1;34m(self, url)\u001b[0m\n\u001b[0;32m    354\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget\u001b[39m(\u001b[38;5;28mself\u001b[39m, url: \u001b[38;5;28mstr\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m    355\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Loads a web page in the current browser session.\"\"\"\u001b[39;00m\n\u001b[1;32m--> 356\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[43mCommand\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mGET\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43murl\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m}\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py:345\u001b[0m, in \u001b[0;36mWebDriver.execute\u001b[1;34m(self, driver_command, params)\u001b[0m\n\u001b[0;32m    342\u001b[0m     \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msessionId\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m params:\n\u001b[0;32m    343\u001b[0m         params[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msessionId\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msession_id\n\u001b[1;32m--> 345\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcommand_executor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdriver_command\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    346\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m response:\n\u001b[0;32m    347\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39merror_handler\u001b[38;5;241m.\u001b[39mcheck_response(response)\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\selenium\\webdriver\\remote\\remote_connection.py:300\u001b[0m, in \u001b[0;36mRemoteConnection.execute\u001b[1;34m(self, command, params)\u001b[0m\n\u001b[0;32m    298\u001b[0m data \u001b[38;5;241m=\u001b[39m utils\u001b[38;5;241m.\u001b[39mdump_json(params)\n\u001b[0;32m    299\u001b[0m url \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_url\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00mpath\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m--> 300\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcommand_info\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\selenium\\webdriver\\remote\\remote_connection.py:321\u001b[0m, in \u001b[0;36mRemoteConnection._request\u001b[1;34m(self, method, url, body)\u001b[0m\n\u001b[0;32m    318\u001b[0m     body \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m    320\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mkeep_alive:\n\u001b[1;32m--> 321\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_conn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    322\u001b[0m     statuscode \u001b[38;5;241m=\u001b[39m response\u001b[38;5;241m.\u001b[39mstatus\n\u001b[0;32m    323\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\urllib3\\request.py:81\u001b[0m, in \u001b[0;36mRequestMethods.request\u001b[1;34m(self, method, url, fields, headers, **urlopen_kw)\u001b[0m\n\u001b[0;32m     77\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrequest_encode_url(\n\u001b[0;32m     78\u001b[0m         method, url, fields\u001b[38;5;241m=\u001b[39mfields, headers\u001b[38;5;241m=\u001b[39mheaders, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39murlopen_kw\n\u001b[0;32m     79\u001b[0m     )\n\u001b[0;32m     80\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m---> 81\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest_encode_body\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m     82\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfields\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfields\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43murlopen_kw\u001b[49m\n\u001b[0;32m     83\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\urllib3\\request.py:173\u001b[0m, in \u001b[0;36mRequestMethods.request_encode_body\u001b[1;34m(self, method, url, fields, headers, encode_multipart, multipart_boundary, **urlopen_kw)\u001b[0m\n\u001b[0;32m    170\u001b[0m extra_kw[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mheaders\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mupdate(headers)\n\u001b[0;32m    171\u001b[0m extra_kw\u001b[38;5;241m.\u001b[39mupdate(urlopen_kw)\n\u001b[1;32m--> 173\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mextra_kw\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\urllib3\\poolmanager.py:376\u001b[0m, in \u001b[0;36mPoolManager.urlopen\u001b[1;34m(self, method, url, redirect, **kw)\u001b[0m\n\u001b[0;32m    374\u001b[0m     response \u001b[38;5;241m=\u001b[39m conn\u001b[38;5;241m.\u001b[39murlopen(method, url, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkw)\n\u001b[0;32m    375\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m--> 376\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mu\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest_uri\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkw\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    378\u001b[0m redirect_location \u001b[38;5;241m=\u001b[39m redirect \u001b[38;5;129;01mand\u001b[39;00m response\u001b[38;5;241m.\u001b[39mget_redirect_location()\n\u001b[0;32m    379\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m redirect_location:\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\urllib3\\connectionpool.py:715\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[1;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[0;32m    712\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_prepare_proxy(conn)\n\u001b[0;32m    714\u001b[0m \u001b[38;5;66;03m# Make the request on the httplib connection object.\u001b[39;00m\n\u001b[1;32m--> 715\u001b[0m httplib_response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    716\u001b[0m \u001b[43m    \u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    717\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    718\u001b[0m \u001b[43m    \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    719\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    720\u001b[0m \u001b[43m    \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    721\u001b[0m \u001b[43m    \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    722\u001b[0m \u001b[43m    \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    723\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    725\u001b[0m \u001b[38;5;66;03m# If we're going to release the connection in ``finally:``, then\u001b[39;00m\n\u001b[0;32m    726\u001b[0m \u001b[38;5;66;03m# the response doesn't need to know about the connection. Otherwise\u001b[39;00m\n\u001b[0;32m    727\u001b[0m \u001b[38;5;66;03m# it will also try to release it and we'll have a double-release\u001b[39;00m\n\u001b[0;32m    728\u001b[0m \u001b[38;5;66;03m# mess.\u001b[39;00m\n\u001b[0;32m    729\u001b[0m response_conn \u001b[38;5;241m=\u001b[39m conn \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m release_conn \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\urllib3\\connectionpool.py:467\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[1;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[0;32m    462\u001b[0m             httplib_response \u001b[38;5;241m=\u001b[39m conn\u001b[38;5;241m.\u001b[39mgetresponse()\n\u001b[0;32m    463\u001b[0m         \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m    464\u001b[0m             \u001b[38;5;66;03m# Remove the TypeError from the exception chain in\u001b[39;00m\n\u001b[0;32m    465\u001b[0m             \u001b[38;5;66;03m# Python 3 (including for exceptions like SystemExit).\u001b[39;00m\n\u001b[0;32m    466\u001b[0m             \u001b[38;5;66;03m# Otherwise it looks like a bug in the code.\u001b[39;00m\n\u001b[1;32m--> 467\u001b[0m             \u001b[43msix\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mraise_from\u001b[49m\u001b[43m(\u001b[49m\u001b[43me\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[0;32m    468\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (SocketTimeout, BaseSSLError, SocketError) \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m    469\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_raise_timeout(err\u001b[38;5;241m=\u001b[39me, url\u001b[38;5;241m=\u001b[39murl, timeout_value\u001b[38;5;241m=\u001b[39mread_timeout)\n",
      "File \u001b[1;32m<string>:3\u001b[0m, in \u001b[0;36mraise_from\u001b[1;34m(value, from_value)\u001b[0m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\urllib3\\connectionpool.py:462\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[1;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[0;32m    459\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m:\n\u001b[0;32m    460\u001b[0m     \u001b[38;5;66;03m# Python 3\u001b[39;00m\n\u001b[0;32m    461\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m--> 462\u001b[0m         httplib_response \u001b[38;5;241m=\u001b[39m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgetresponse\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    463\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m    464\u001b[0m         \u001b[38;5;66;03m# Remove the TypeError from the exception chain in\u001b[39;00m\n\u001b[0;32m    465\u001b[0m         \u001b[38;5;66;03m# Python 3 (including for exceptions like SystemExit).\u001b[39;00m\n\u001b[0;32m    466\u001b[0m         \u001b[38;5;66;03m# Otherwise it looks like a bug in the code.\u001b[39;00m\n\u001b[0;32m    467\u001b[0m         six\u001b[38;5;241m.\u001b[39mraise_from(e, \u001b[38;5;28;01mNone\u001b[39;00m)\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\http\\client.py:1378\u001b[0m, in \u001b[0;36mHTTPConnection.getresponse\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   1376\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m   1377\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m-> 1378\u001b[0m         \u001b[43mresponse\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbegin\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1379\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m:\n\u001b[0;32m   1380\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclose()\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\http\\client.py:318\u001b[0m, in \u001b[0;36mHTTPResponse.begin\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    316\u001b[0m \u001b[38;5;66;03m# read until we get a non-100 response\u001b[39;00m\n\u001b[0;32m    317\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[1;32m--> 318\u001b[0m     version, status, reason \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_read_status\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    319\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m status \u001b[38;5;241m!=\u001b[39m CONTINUE:\n\u001b[0;32m    320\u001b[0m         \u001b[38;5;28;01mbreak\u001b[39;00m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\http\\client.py:279\u001b[0m, in \u001b[0;36mHTTPResponse._read_status\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    278\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_read_status\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m--> 279\u001b[0m     line \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mstr\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfp\u001b[38;5;241m.\u001b[39mreadline(_MAXLINE \u001b[38;5;241m+\u001b[39m \u001b[38;5;241m1\u001b[39m), \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124miso-8859-1\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m    280\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(line) \u001b[38;5;241m>\u001b[39m _MAXLINE:\n\u001b[0;32m    281\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m LineTooLong(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstatus line\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\socket.py:706\u001b[0m, in \u001b[0;36mSocketIO.readinto\u001b[1;34m(self, b)\u001b[0m\n\u001b[0;32m    704\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m    705\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m--> 706\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrecv_into\u001b[49m\u001b[43m(\u001b[49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    707\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m timeout:\n\u001b[0;32m    708\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_timeout_occurred \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "isDownLoadTrue = True\n",
    "isimgTrue = True\n",
    "\n",
    "for i in range(1,len(url_list)):\n",
    "    print(\"------------------开始下一篇文章--------------------\")\n",
    "    print(\"此篇文章：i=\",i)\n",
    "    isDownLoadTrue = True\n",
    "    driver = webdriver.Chrome(options=opts) \n",
    "    time.sleep(3)\n",
    "    driver.get(url_list[i])\n",
    "    timeout()\n",
    "    print(url_list[i])\n",
    "    time.sleep(5)\n",
    "    timeout()\n",
    "    hov_file_nmb = check_file()\n",
    "    print(\"hov_file_nmb:\",hov_file_nmb)\n",
    "    down_PDF()\n",
    "    timeout()\n",
    "    file_nmb = check_file()\n",
    "    print(\"file_nmb:\",file_nmb)\n",
    "    print(\"isDownLoadTrue：\",isDownLoadTrue)\n",
    "    while isDownLoadTrue:        \n",
    "        if file_nmb>hov_file_nmb:\n",
    "            print(\"下载成功，文章数量增加\")\n",
    "            i += 1\n",
    "            isDownLoadTrue = False\n",
    "            print(\"关闭页面，1.i+1\")\n",
    "            driver.close()#\"关掉页面\"   \n",
    "        else:\n",
    "            print(\"文章数量未增加\")\n",
    "            switch_win()\n",
    "            print(\"isimgTrue：\",isimgTrue)\n",
    "            while isimgTrue:\n",
    "                switch_win()\n",
    "                if img_isTrue():\n",
    "                    hov_file_nmb = check_file()\n",
    "                    print(\"验证码前hov_file_nmb:\",hov_file_nmb)\n",
    "                    results = OCR()\n",
    "                    file_nmb = check_file()\n",
    "                    print(\"验证码后file_nmb:\",file_nmb)\n",
    "                    if results == True:\n",
    "                            if img_isTrue()==False:\n",
    "                                print(\"并发or验证码错误过多or出现弹窗——关闭当前页面-切换窗口-重新点击下载PDF\")\n",
    "                                driver.close()#\"关掉页面\" \n",
    "                                i = i\n",
    "                                print(\"{i=:}\",i)            \n",
    "                            else:\n",
    "                                if file_nmb>hov_file_nmb:\n",
    "                                    print(\"下载成功，文章数量增加\")\n",
    "                                    i += 1\n",
    "                                    isDownLoadTrue = False\n",
    "                                    print(\"关闭页面，2.i+1\")\n",
    "                                    driver.close()#\"关掉页面\"    \n",
    "                                else:\n",
    "                                    print(\"换验证码\")\n",
    "                                    driver.find_element_by_xpath('//*[@id=\"vImg\"]').click() ##换验证码\n",
    "                                    print(\"文章数量未增加\") \n",
    "                                    isimgTrue = True\n",
    "                        \n",
    "                    else:\n",
    "                        #验证码错误or出现弹窗——关闭当前页面-切换窗口-重新点击下载PDF\n",
    "                        isimgTrue = True                        \n",
    "                else:\n",
    "                    print(\"关闭页面\")\n",
    "                    driver.close()#\"关掉页面\"                    \n",
    "                    print(\"【i=】\",i)\n",
    "                break\n",
    "            else:            # else2\n",
    "                continue\n",
    "            break            # break2\n",
    "                    \n",
    "\n",
    "                    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "element = driver.find_element_by_xpath('//html')\n",
    "main_content =element.get_attribute('outerHTML')  \n",
    "html = requests_html.HTML(html= main_content, url='https://localhost/')\n",
    "data_list = html.xpath('//*[@id=\"vImg\"]')\n",
    "data_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "164.988px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
